{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed = []\n",
    "#this script aggregates the trials into one single file that can be analyzed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying gs://sahai_echo/20190814_074239/1179.npy [Content-Type=application/octet-stream]...\n",
      "Removing gs://sahai_echo/20190814_074239/1179.npy...                            \n",
      "Copying gs://sahai_echo/20190814_074239/1476.npy [Content-Type=application/octet-stream]...\n",
      "Removing gs://sahai_echo/20190814_074239/1476.npy...                            \n",
      "\n",
      "Operation completed over 2 objects/6.5 MiB.                                      \n"
     ]
    }
   ],
   "source": [
    "#move from the job bucket to the processing bucket\n",
    "!gsutil mv -p gs://sahai_echo/20190814_074239/*.npy gs://torch-echo/results/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "import google.datalab.storage as storage\n",
    "bucket_name='torch-echo'\n",
    "bucket = storage.Bucket(bucket_name)\n",
    "assert bucket.exists()\n",
    "\n",
    "results =[]\n",
    "\n",
    "for obj in bucket.objects():\n",
    "    if \".npy\" in obj.key and \\\n",
    "        \"results/\" in obj.key and \\\n",
    "        int(obj.key[8:-4]) not in processed and \\\n",
    "        'QPSK' not in obj.key:\n",
    "        results += [obj]\n",
    "\n",
    "tasks = list(map(lambda r: int(r.key.split(\"/\")[-1].split(\".\")[0]), results))\n",
    "# tasks += list(map(lambda k: int(k.split(\"/\")[-1].split(\".\")[0]), processed))\n",
    "print(len(tasks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "( 1179 1476 )"
     ]
    }
   ],
   "source": [
    "missing = []\n",
    "\n",
    "print(\"(\", end=\" \")\n",
    "for i in range(max(tasks)+1):\n",
    "    if i not in tasks:\n",
    "        print(i, end=\" \")\n",
    "    missing += [i]\n",
    "    \n",
    "print(\")\", end=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 .1151 .1152 .1153 .1154 .1155 .1156 .1157 .1158 .1159 .1160 .1161 .1162 .1163 .1164 .1165 .1166 .1167 .1168 .1169 .1170 .1171 .1172 .1173 .1174 .1175 .1176 .1177 .1178 .1179 .1180 .1181 .1182 .1183 .1184 .1185 .1186 .1187 .1188 .1189 .1190 .1191 .1192 .1193 .1194 .1195 .1196 .1197 .1198 .1199 .1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 .writing...\n",
      "./shared_preamble-QPSK_neural_vs_clone-4_2.npy False\n",
      "False\n",
      "1451 .1452 .1453 .1454 .1455 .1456 .1457 .1458 .1459 .1460 .1461 .1462 .1463 .1464 .1465 .1466 .1467 .1468 .1469 .1470 .1471 .1472 .1473 .1474 .1475 .1476 .1477 .1478 .1479 .1480 .1481 .1482 .1483 .1484 .1485 .1486 .1487 .1488 .1489 .1490 .1491 .1492 .1493 .1494 .1495 .1496 .1497 .1498 .1499 .1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 "
     ]
    }
   ],
   "source": [
    "from io import BytesIO, StringIO\n",
    "import numpy as np\n",
    "import os\n",
    "modified_tasks = list(range(1120,1230)) + list(range(1420,1530))\n",
    "match = ['private_preamble-QPSK_neural_vs_classic-4_2' , 'shared_preamble-QPSK_neural_vs_clone-4_2']\n",
    "tasks = modified_tasks\n",
    "def write(experiments):\n",
    "  #write the chunk out\n",
    "  print(\"writing...\")\n",
    "  n = 0\n",
    "  for identifier in experiments.keys():\n",
    "    print(\"./%s.npy\"%(identifier), os.path.isfile(\"./%s.npy\"%(identifier)))\n",
    "    trials = experiments[identifier]\n",
    "    trials_npy = bucket.object(\"processed/%s.npy\"%(identifier))\n",
    "    print(trials_npy.exists())\n",
    "    if os.path.isfile(\"./%s.npy\"%(identifier)):\n",
    "      saved_trials = np.load(\"%s.npy\"%(identifier)).item()\n",
    "      trials = {**saved_trials, **trials}\n",
    "    elif trials_npy.exists():\n",
    "      uri = trials_npy.uri\n",
    "      %gcs read --object $uri --variable data\n",
    "      saved_trials = np.load(BytesIO(data)).item()\n",
    "      trials = {**saved_trials, **trials}\n",
    "    np.save(\"./%s.npy\"%(identifier), trials)\n",
    "    n += 1\n",
    "  return n\n",
    "\n",
    "experiments = {}\n",
    "current = None\n",
    "to_transfer = 0 \n",
    "for t in sorted(tasks):\n",
    "  if to_transfer > 10:\n",
    "    print(\"transferring...\")\n",
    "    !gsutil -m mv -p  ./*.npy gs://torch-echo/processed\n",
    "    to_transfer = 0\n",
    "  result_npy = bucket.object('results/%i.npy'%t)\n",
    "  print(t, end=\" \")\n",
    "  processed += [t]\n",
    "  uri = result_npy.uri\n",
    "  %gcs read --object $uri --variable data\n",
    "  result_array = np.load(BytesIO(data))\n",
    "  meta, result = result_array[0], result_array[1:]\n",
    "  experiment_name = meta['experiment_name']\n",
    "  protocol = meta['protocol']  \n",
    "  train_snr = str(meta['train_SNR_db']).replace(\".\",\"_\")\n",
    "  trial_num = meta['trial_num']\n",
    "  identifier = \"-\".join([protocol, experiment_name, train_snr])\n",
    "  if identifier not in match:\n",
    "    continue\n",
    "  print(\".\", end=\"\")\n",
    "  if current == None:\n",
    "    current = experiment_name\n",
    "  elif current != experiment_name:\n",
    "    to_transfer += write(experiments)\n",
    "    experiments = {}\n",
    "    current = experiment_name\n",
    "  trials = experiments.get(identifier, {})\n",
    "  trials[trial_num] = {\n",
    "    'meta' : meta,\n",
    "    'data': result\n",
    "  }\n",
    "  experiments[identifier] = trials\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "writing...\n",
      "./private_preamble-QPSK_neural_vs_classic-4_2.npy False\n",
      "False\n",
      "Copying file://./private_preamble-QPSK_neural_vs_classic-4_2.npy [Content-Type=application/octet-stream]...\n",
      "==> NOTE: You are uploading one or more large file(s), which would run\n",
      "significantly faster if you enable parallel composite uploads. This\n",
      "feature can be enabled by editing the\n",
      "\"parallel_composite_upload_threshold\" value in your .boto\n",
      "configuration file. However, note that if you do this large files will\n",
      "be uploaded as `composite objects\n",
      "<https://cloud.google.com/storage/docs/composite-objects>`_,which\n",
      "means that any user who downloads such objects will need to have a\n",
      "compiled crcmod installed (see \"gsutil help crcmod\"). This is because\n",
      "without a compiled crcmod, computing checksums on composite objects is\n",
      "so slow that gsutil disables downloads of composite objects.\n",
      "\n",
      "Copying file://./shared_preamble-QPSK_neural_vs_clone-4_2.npy [Content-Type=application/octet-stream]...\n",
      "Removing file://./private_preamble-QPSK_neural_vs_classic-4_2.npy...            \n",
      "Removing file://./shared_preamble-QPSK_neural_vs_clone-4_2.npy...               \n",
      "- [2/2 files][323.6 MiB/323.6 MiB] 100% Done                                    \n",
      "Operation completed over 2 objects/323.6 MiB.                                    \n"
     ]
    }
   ],
   "source": [
    "#last\n",
    "write(experiments)\n",
    "!gsutil -m mv -p  ./*.npy gs://torch-echo/processed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 31]) {'data': array([{'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 1, 'db_off': [4.999999999999982, 4.799999999999983, 4.399999999999984, 3.999999999999986, 3.1999999999999886, 1.5999999999999943], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.00581, 0.0208, 0.0372, 0.07802, 0.14279, 0.32178], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.002905, 0.0104, 0.0186, 0.03901, 0.07147, 0.16685]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 8, 'db_off': [1.399999999999995, 1.1999999999999957, 1.1999999999999957, 0.9999999999999964, 0.7999999999999972, 0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [1e-05, 0.00025, 0.0012, 0.00691, 0.0334, 0.21452], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [5e-06, 0.000125, 0.0006, 0.00346, 0.01682, 0.113105]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 15, 'db_off': [0.0, 0.0, 0.0, 0.0, -0.1999999999999993, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.00017, 0.00198, 0.01672, 0.18856], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 8.5e-05, 0.00099, 0.00838, 0.099115]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 22, 'db_off': [0.0, 0.0, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.0002, 0.0019, 0.01816, 0.19095], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 0.0001, 0.00095, 0.00912, 0.100515]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 29, 'db_off': [0.0, 0.0, 0.0, -0.1999999999999993, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.00015, 0.00169, 0.01798, 0.18977], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 7.5e-05, 0.000845, 0.00902, 0.09996]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 36, 'db_off': [0.0, 0.0, -0.1999999999999993, 0.0, -0.1999999999999993, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.00013, 0.00198, 0.01674, 0.18857], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 6.5e-05, 0.00099, 0.0084, 0.09929]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 43, 'db_off': [0.0, -1.5999999999999943, 0.0, -0.1999999999999993, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 0.0, 0.00016, 0.00166, 0.01706, 0.18941], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 0.0, 8e-05, 0.00083, 0.008585, 0.09967]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 50, 'db_off': [0.0, -1.5999999999999943, -0.1999999999999993, 0.0, -0.1999999999999993, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 0.0, 0.0001, 0.00202, 0.01658, 0.19089], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 0.0, 5e-05, 0.00101, 0.00833, 0.10056]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 57, 'db_off': [0.0, 0.3999999999999986, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 5e-05, 0.00017, 0.00188, 0.01713, 0.18952], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 2.5e-05, 8.5e-05, 0.00094, 0.008635, 0.099525]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 64, 'db_off': [0.0, 0.0, 0.0, 0.0, -0.1999999999999993, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.00014, 0.00186, 0.01612, 0.18799], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 7e-05, 0.00093, 0.008135, 0.098885]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 71, 'db_off': [0.0, 0.1999999999999993, 0.1999999999999993, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 3e-05, 0.00026, 0.00193, 0.01724, 0.19103], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1.5e-05, 0.00013, 0.000965, 0.00866, 0.100545]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 78, 'db_off': [0.0, -0.1999999999999993, -0.1999999999999993, -0.1999999999999993, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 1e-05, 0.00013, 0.00163, 0.01739, 0.18875], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 5e-06, 6.5e-05, 0.000815, 0.00875, 0.099175]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 85, 'db_off': [0.0, -0.1999999999999993, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 1e-05, 0.00018, 0.00226, 0.0179, 0.19091], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 5e-06, 9e-05, 0.001135, 0.008985, 0.100695]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 92, 'db_off': [0.0, -0.1999999999999993, 0.0, -0.1999999999999993, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 1e-05, 0.00015, 0.00172, 0.01725, 0.18816], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 5e-06, 7.5e-05, 0.00086, 0.00867, 0.099115]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 99, 'db_off': [0.0, 0.0, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.00017, 0.00188, 0.01694, 0.18994], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 8.5e-05, 0.00094, 0.00853, 0.099845]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 106, 'db_off': [0.0, -1.5999999999999943, -0.1999999999999993, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 0.0, 0.00012, 0.00185, 0.01693, 0.18869], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 0.0, 6e-05, 0.000925, 0.008515, 0.099135]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 113, 'db_off': [0.0, -1.5999999999999943, -0.3999999999999986, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 0.0, 6e-05, 0.00192, 0.01699, 0.18798], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 0.0, 3e-05, 0.00096, 0.00853, 0.098875]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 120, 'db_off': [0.0, 0.1999999999999993, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 4e-05, 0.00018, 0.0019, 0.01729, 0.18996], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 2e-05, 9e-05, 0.00095, 0.00867, 0.0998]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 127, 'db_off': [0.0, 0.0, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 2e-05, 0.0002, 0.00189, 0.0174, 0.18903], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1e-05, 0.0001, 0.000945, 0.008725, 0.099595]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 134, 'db_off': [0.0, 0.1999999999999993, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 4e-05, 0.00014, 0.002, 0.01714, 0.18763], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 2e-05, 7e-05, 0.001, 0.00865, 0.09847]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 141, 'db_off': [0.0, 0.1999999999999993, 0.1999999999999993, 0.0, -0.1999999999999993, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 4e-05, 0.00021, 0.00188, 0.01672, 0.19086], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 2e-05, 0.000105, 0.000945, 0.00838, 0.10038]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 148, 'db_off': [0.0, -0.1999999999999993, 0.0, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 1e-05, 0.00016, 0.00192, 0.01737, 0.18821], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 5e-06, 8e-05, 0.00096, 0.00871, 0.099095]},\n",
      "       {'demod_grid_1': array([3, 3, 3, ..., 0, 0, 0]), 'batches_sent': 150, 'db_off': [0.0, 0.1999999999999993, -0.3999999999999986, 0.0, 0.0, -0.1999999999999993], 'mod_std_1': [0.0, 0.0], 'test_sers': [0.0, 3e-05, 9e-05, 0.00208, 0.01708, 0.18879], 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'constellation_1': array([[ 0.70710677,  0.70710677],\n",
      "       [-0.70710677,  0.70710677],\n",
      "       [ 0.70710677, -0.70710677],\n",
      "       [-0.70710677, -0.70710677]], dtype=float32), 'test_bers': [0.0, 1.5e-05, 4.5e-05, 0.00104, 0.008565, 0.099345]}],\n",
      "      dtype=object), 'meta': {'num_agents': 1, 'early_stop': False, 'num_results': 23, 'bits_per_symbol': 2, 'early_stop_threshold_db_off': 0.1, 'trial_num': 32, 'train_SNR_db': 13, 'test_SNR_dbs': [14.599999999999913, 12.999999999999918, 11.999999999999922, 10.399999999999928, 8.399999999999935, 4.1999999999999496], 'experiment_name': 'QPSK_classic_and_neural', 'batch_size': 32, 'protocol': 'gradient_passing'}}\n"
     ]
    }
   ],
   "source": [
    "# test_npy = bucket.object('processed/gradient_passing-QPSK_classic_and_neural-13.npy')\n",
    "# uri = test_npy.uri\n",
    "# %gcs read --object $uri --variable data\n",
    "# test = np.load(BytesIO(data)).item()\n",
    "# print(test.keys(), test[32])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.isfile(\"./shared_preamble-QPSK_neural_vs_clone-4_2.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
